package com.study.bigdata.spark.core.rdd.dep

import org.apache.spark.{SparkConf, SparkContext}

object Scala02_WordCount {
  def main(args: Array[String]): Unit = {

    val conf = new SparkConf().setMaster("local[*]").setAppName("WordCount")
    val sc = new SparkContext(conf)

    val lines = sc.textFile("data/word.txt")
    println(lines.dependencies)//OneToOneDependency
    println("******************************")

    val words = lines.flatMap(_.split(" "))
    println(words.dependencies)//OneToOneDependency
    println("******************************")

    val wordToOne = words.map((_,1))
    println(wordToOne.dependencies)//OneToOneDependency
    println("******************************")

    val wordCount = wordToOne.reduceByKey(_+_)
    println(wordCount.dependencies)//ShuffleDependency
    println("******************************")

    wordCount.collect().foreach(println)
    // 依赖分为窄依赖和宽依赖
    sc.stop()

  }

}
